package exp;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Vector;

import edu.hit.irlab.nlp.stopwords.StopWords;

import opennlp.tools.tokenize.Tokenizer;
import tools.FileReaderEnhance;
import tools.FileWriterEnhance;
import tools.ObjectTools;
import tools.nlp.TokenizerSimple;
import tools.similarity.Cosine;
import tools.twitter.TweetTidy;
import tools.twitter.URLTools;

/**
 * Filtering of tweets using titles of corresponding urls
 * @author Lanjun
 *
 */
public class EXP7 {
	static Vector<String> sw = (new StopWords("data/stopwords.dat")).getStopwords();
	static HashMap<String, Boolean> stopwords = new HashMap<String, Boolean>();
	static {
		for (String stopword : sw) {
			stopwords.put(stopword, true);
		}
	}
	
	public void run(String tPath, String hPath, double SIMTHRESHOLD, String output) {
		for (File file : (new File(tPath)).listFiles()) {
			String[] lines = FileReaderEnhance.readToString(file, "UTF-8")
					.split("\n");
			
			String hl = hPath + file.getName();
			@SuppressWarnings("unchecked")
			HashMap<String, URLContent> hls = (HashMap<String, URLContent>) ObjectTools.readFromFile(hl);
			HashMap<String, URLContent> map = new HashMap<>();
			for (URLContent temp : hls.values()) {
				for (String url : temp.shortURL) {
					map.put(url, temp);
				}
			}
			
			TokenizerSimple ts = new TokenizerSimple();
			Tokenizer tokenizer = ts.getTokenizer();
			
			ArrayList<String> preserved = new ArrayList<>();
			
			for (String line : lines) {
//				System.out.println(line);
				String [] elements = line.split("\t~\t");
				String tweet = elements[0];
				tweet = TweetTidy.doTidyHTML(tweet);
				ArrayList<String> urls = URLTools.getURLs(tweet);
				if (urls.size() == 0) {
					preserved.add(tweet);
				} else if (urls.size() == 1) {
					String url = urls.get(0);
					if (map.containsKey(url)) {
						String title = map.get(url).title;
						if (title == null) {
							// the link is expired or without title
							preserved.add(tweet);
							continue;
						}
						String tokens1[] = tokenizer.tokenize(TweetTidy.doTidyAll(line));
						String tokens2[] = tokenizer.tokenize(TweetTidy.doTidyAll(title));
						
						//Judge if title contains all words from the line			
						if (contains(tokens1, tokens2)) {
							continue;
						}
						
						//if not, calculate the cosine similarity
						if (Cosine.getCosine(tokens1, tokens2) < SIMTHRESHOLD) {
							preserved.add(tweet);
						}
					} else {
						// no such URL. Often caused by multiple short url linked to the same final URL 
						// or this url is dead. Can not be downloaded
						System.err.println("Not found:" + url);
					}
				} else {
					// multiple urls in a single tweet
					preserved.add(tweet);
				}
			}
			
			StringBuilder sb =  new StringBuilder();
			for (String tweet : preserved) {
				sb.append(tweet);
				sb.append('\n');
			}
			
			FileWriterEnhance fwe = new FileWriterEnhance(output + file.getName(), "UTF-8");
			fwe.WriteToFile(sb.toString());
		}
	}
	
	/**
	 * Judge if tokens2 contains all tokens from tokens1
	 * @param tokens1
	 * @param tokens2
	 * @return true if so
	 */
	public static boolean contains(String[] tokens1, String[] tokens2) {
		HashMap<String, Boolean> tempMap = new HashMap<>();
		boolean flag = true;
		
		for (String temp : tokens2) {
			tempMap.put(temp.toLowerCase(), true);
		}
		for (String temp : tokens1) {
			temp = temp.toLowerCase();
			if (stopwords.containsKey(temp) || temp.equals("RT")) {
				continue;
			}
			if (!tempMap.containsKey(temp)) {
				flag = false;
				break;
			}
		}
		return flag;
	}
	
	public static void main(String[] args) {
		try {
			System.setOut(new PrintStream(new File("./logs/EXP7.out.log")));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		
		double SIMTHRESHOLD = 0.70;
		String tPath = "data/_newData/plainText/";
		String hPath = "data/_newData/obj/highlights/";
		String output = "data/_newData/plainText_filtered/";
		EXP7 exp7 = new EXP7();
		exp7.run(tPath, hPath, SIMTHRESHOLD, output);
	}
}
